import sys

inputFileName = "/data1/team/weijiang/machine-learning-project-related/learningToPrune/learningToPruneIntermediateTrainingOutputFile.txt"
inputFileHandler =file(inputFileName, 'r')

outputFileName = "/data1/team/weijiang/machine-learning-project-related/learningToPrune/learningToPruneIntermediateTrainingOutputFile2.arff"
outFileHandler =file(outputFileName, 'w')

outFileHandler.write("@relation postings" + "\n")
outFileHandler.write("\n")
outFileHandler.write("@attribute termID { ")
for i in range(0,123):
    outFileHandler.write(str(i) + " ")
outFileHandler.write("}" + "\n")
outFileHandler.write("@attribute docID real" + "\n")
outFileHandler.write("@attribute bm25 real" + "\n")
outFileHandler.write("@attribute postingLabel { Top20, NotTop1000 }" + "\n")
outFileHandler.write("\n")
outFileHandler.write("@data" + "\n")

currentLine = ""
indexQueryTermDict = {}
while True:
    #The new section begins, so let's do each content's parsing based on order.
    #print "At the beginning of ONE section."
    queryTermRelativePostionIndexDict = {}
    while currentLine == "":
        currentLine = inputFileHandler.readline().strip()
    #print "currentLineCheck:",currentLine
    elements = currentLine.strip().split(" ")
    queryID = elements[0]
    queryProcessingSemantics = elements[1]
    for element in elements[2:]:
        queryTerm = element.split(":")[0]
        queryTermPositionIndex = int(element.split(":")[1])
        if queryTerm not in indexQueryTermDict:
            indexQueryTermDict[queryTerm] = len(indexQueryTermDict)
        if queryTermPositionIndex not in queryTermRelativePostionIndexDict:
            queryTermRelativePostionIndexDict[queryTermPositionIndex] = queryTerm
    
    #print "queryTermRelativePostionIndexDict:",queryTermRelativePostionIndexDict
    #print "indexQueryTermDict:",indexQueryTermDict
    #print "queryID:",queryID, "queryProcessingSemantics:",queryProcessingSemantics, "indexQueryTermDict:",indexQueryTermDict
    
    tempCounter = 0
    positveLabelUpperBound = 20
    negativeLabelLowerBound = 1000
    
    while currentLine != "":
        currentLine = inputFileHandler.readline()
        if currentLine.strip() != "":
            elements = currentLine.strip().split(" ")
            if len(elements) == 13:
                positionRanking = int(elements[0])
                if positionRanking <= positveLabelUpperBound:
                    exampleLabel = "Top" + str(positveLabelUpperBound)
                    docID = int(elements[12])
                    for positionIndex,element in enumerate(elements[ 1:len(queryTermRelativePostionIndexDict)+1 ]):
                        currentQueryTerm = queryTermRelativePostionIndexDict[positionIndex]
                        trainingExampleLine = str(indexQueryTermDict[currentQueryTerm]) + " " + str(docID) + " " + elements[1:len(indexQueryTermDict)+1][positionIndex] + " " + exampleLabel
                        print trainingExampleLine
                        outFileHandler.write(trainingExampleLine + "\n")
                elif positionRanking > negativeLabelLowerBound:
                    exampleLabel = "NotTop" + str(negativeLabelLowerBound)
                    docID = int(elements[12])
                    for positionIndex,element in enumerate(elements[ 1:len(queryTermRelativePostionIndexDict)+1 ]):    
                        currentQueryTerm = queryTermRelativePostionIndexDict[positionIndex]
                        trainingExampleLine = str(indexQueryTermDict[currentQueryTerm]) + " " + str(docID) + " " + elements[1:len(indexQueryTermDict)+1][positionIndex] + " " + exampleLabel
                        print trainingExampleLine
                        outFileHandler.write(trainingExampleLine + "\n")
                else:
                    pass
                tempCounter += 1
            else:
                if len(elements) == 3:
                    numOfResultsReturned = int(elements[0])
                    numOfResultsInTotal = int(elements[1])
                    elasedTimeInTotal = float(elements[2])
                    #print numOfResultsReturned,numOfResultsInTotal,elasedTimeInTotal
                    currentLine = inputFileHandler.readline().strip()
                    #print "At the end of ONE section."
                    break
                else:
                    print "Vital Error. Mark1"
                    sys.exit()
        else:
            pass

outFileHandler.close()   
                
  
    